LP#1744385: Make use of short-term unAPI cache and parallelize metarecord constituent...

Signed-off-by: Mike Rylander <mrylander@gmail.com>
Signed-off-by: Kathy Lussier <klussier@masslnc.org>
Signed-off-by: Dan Wells <dbw2@calvin.edu>

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Search.pm b/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Search.pm
index b95ffc0..c21566a 100644 (file)
--- a/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Search.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Search.pm
@@ -525,33 +525,43 @@ sub load_rresults {
     # load temporary_list settings for user and ou:
     $self->_load_lists_and_settings if ($ctx->{user});
 
+    my %mr_contents;
     # shove recs into context in search results order
     for my $rec_id (@$rec_ids) {
         my ($rec) = grep { $_->{$id_key} == $rec_id } @data;
         push(@{$ctx->{records}}, $rec);
 
         if ($is_meta) {
-            my $meta_results;
             try {
                 my $method = 'open-ils.search.biblio.multiclass.query';
                 $method .= '.staff' if $ctx->{is_staff};
                 my $ses = OpenSRF::AppSession->create('open-ils.search');
-                $self->timelog("Firing off the multiclass query");
+                $self->timelog("Firing off the multiclass query: ". $rec_id);
                 $args->{from_metarecord} = $rec_id;
                 # offset of main search does not apply to the MR
                 # constituents query
                 my $save_offset = $args->{offset};
                 $args->{offset} = 0;
-                my $req = $ses->request($method, $args, $query, 1);
+                $mr_contents{$rec_id} = $ses->request($method, $args, $query, 1);
                 $args->{offset} = $save_offset;
-                $meta_results = $req->gather(1);
-                $self->timelog("Returned from the multiclass query");
-
             } catch Error with {
                 my $err = shift;
                 $logger->error("multiclass search error: $err");
-                $meta_results = {count => 0, ids => []};
             };
+        }
+    }
+
+    # above we fire all searches in parallel, now we collect the results
+    if ($is_meta) {
+        for my $rec_id (@$rec_ids) {
+            my ($rec) = grep { $_->{$id_key} == $rec_id } @data;
+            my $meta_results;
+            if ($mr_contents{$rec_id}) {
+                $meta_results = $mr_contents{$rec_id}->gather(1);
+                $self->timelog("Returned from the multiclass query: ". $rec_id);
+            } else {
+                $meta_results = {count => 0, ids => []};
+            }
             my $meta_rec_ids = [map { $_->[0] } @{$meta_results->{ids}}];
             $rec->{mr_constituent_count} = $meta_results->{count};
             $rec->{mr_constituent_ids} = $meta_rec_ids;

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Util.pm b/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Util.pm
index 5505cd6..7ecddc8 100644 (file)
--- a/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Util.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Util.pm
@@ -398,26 +398,90 @@ sub get_records_and_facets {
         } else {
             $unapi_data->{marc_xml} = XML::LibXML->new->parse_string($unapi_data->{marc_xml})->documentElement;
             $tmp_data{$unapi_data->{id}} = $unapi_data;
+            $unapi_cache->put_cache($unapi_cache_key, { running => $$ }, 5);
         }
     }
 
-    my $unapi_req = $ses->request(
-        'open-ils.cstore.json_query',
-         {from => [
-            $unapi_type, '{'.join(',',@loop_recs).'}', 'marcxml', $flesh,
-            $unapi_args->{site}, 
-            $unapi_args->{depth}, 
-            $slimit,
-            undef, undef, $unapi_args->{pref_lib}
-        ]}
-    );
-
     my $hl_req = $hl_ses->request(
         'open-ils.search.fetch.metabib.display_field.highlight.atomic',
         $self->ctx->{query_struct}{additional_data}{highlight_map},
         @$rec_ids
     ) if (!$is_meta);
 
+    if (@loop_recs) {
+        my $unapi_req = $ses->request(
+            'open-ils.cstore.json_query',
+             {from => [
+                $unapi_type, '{'.join(',',@loop_recs).'}', 'marcxml', $flesh,
+                $unapi_args->{site}, 
+                $unapi_args->{depth}, 
+                $slimit,
+                undef, undef, $unapi_args->{pref_lib}
+            ]}
+        );
+    
+        my $data = $unapi_req->gather(1);
+    
+        $outer_self->timelog("get_records_and_facets(): got feed content");
+    
+        # Protect against requests for non-existent records
+        return unless ($data->{$unapi_type});
+    
+        my $doc = XML::LibXML->new->parse_string($data->{$unapi_type})->documentElement;
+    
+        $outer_self->timelog("get_records_and_facets(): parsed xml");
+        for my $xml ($doc->getElementsByTagName('record')) {
+            $xml = XML::LibXML->new->parse_string($xml->toString)->documentElement;
+    
+            # Protect against legacy invalid MARCXML that might not have a 901c
+            my $bre_id;
+            my $mmr_id;
+            my $bre_id_nodes =  $xml->find('*[@tag="901"]/*[@code="c"]');
+            if ($bre_id_nodes) {
+                $bre_id =  $bre_id_nodes->[0]->textContent;
+            } else {
+                $logger->warn("Missing 901 subfield 'c' in " . $xml->toString());
+            }
+        
+            if ($is_meta) {
+                # extract metarecord ID from mmr.unapi tag
+                for my $node ($xml->getElementsByTagName('abbr')) {
+                    my $title = $node->getAttribute('title');
+                    ($mmr_id = $title) =~ 
+                        s/tag:open-ils.org:U2\@mmr\/(\d+)\/.*/$1/g;
+                    last if $mmr_id;
+                }
+            }
+        
+            my $rec_id = $mmr_id ? $mmr_id : $bre_id;
+            $tmp_data{$rec_id} = {
+                id => $rec_id, 
+                bre_id => $bre_id, 
+                mmr_id => $mmr_id,
+                marc_xml => $xml
+            };
+        
+            if ($rec_id) {
+                # Let other backends grab our data now that we're done.
+                my $key = 'TPAC_unapi_cache_'.$rec_id.'_'.$unapi_cache_key_suffix;
+                my $cache_data = $unapi_cache->get_cache($key);
+                if (!$cache_data || $$cache_data{running} == $$) {
+                    $unapi_cache->put_cache($key, {
+                        bre_id => $bre_id,
+                        mmr_id => $mmr_id,
+                        id => $rec_id, 
+                        marc_xml => $xml->toString
+                    }, 10);
+                }
+            }
+        }
+    }
+
+    if (!$is_meta) {
+        my $hl_data = $hl_req->gather(1); # list of arrayref of hashrefs
+        $self->ctx->{_hl_data} = { map { ''.$$_[0]{source} => $_ } @$hl_data };
+        $outer_self->timelog("get_records_and_facets(): got highlighting content (". keys(%{$self->ctx->{_hl_data}}).")");
+    }
 
     my $facets = {};
     if ($facet_req) {
@@ -451,67 +515,6 @@ sub get_records_and_facets {
     }
     $search->kill_me;
 
-    my $data = $unapi_req->gather(1);
-
-    $outer_self->timelog("get_records_and_facets(): got feed content");
-
-    if (!$is_meta) {
-        my $hl_data = $hl_req->gather(1); # list of arrayref of hashrefs
-        $self->ctx->{_hl_data} = { map { ''.$$_[0]{source} => $_ } @$hl_data };
-        $outer_self->timelog("get_records_and_facets(): got highlighting content (". keys(%{$self->ctx->{_hl_data}}).")");
-    }
-
-    # Protect against requests for non-existent records
-    return unless $data->{$unapi_type};
-
-    my $doc = XML::LibXML->new->parse_string($data->{$unapi_type})->documentElement;
-
-    $outer_self->timelog("get_records_and_facets(): parsed xml");
-    for my $xml ($doc->getElementsByTagName('record')) {
-        $xml = XML::LibXML->new->parse_string($xml->toString)->documentElement;
-
-        # Protect against legacy invalid MARCXML that might not have a 901c
-        my $bre_id;
-        my $mmr_id;
-        my $bre_id_nodes =  $xml->find('*[@tag="901"]/*[@code="c"]');
-        if ($bre_id_nodes) {
-            $bre_id =  $bre_id_nodes->[0]->textContent;
-        } else {
-            $logger->warn("Missing 901 subfield 'c' in " . $xml->toString());
-        }
-    
-        if ($is_meta) {
-            # extract metarecord ID from mmr.unapi tag
-            for my $node ($xml->getElementsByTagName('abbr')) {
-                my $title = $node->getAttribute('title');
-                ($mmr_id = $title) =~ 
-                    s/tag:open-ils.org:U2\@mmr\/(\d+)\/.*/$1/g;
-                last if $mmr_id;
-            }
-        }
-    
-        my $rec_id = $mmr_id ? $mmr_id : $bre_id;
-        $tmp_data{$rec_id} = {
-            id => $rec_id, 
-            bre_id => $bre_id, 
-            mmr_id => $mmr_id,
-            marc_xml => $xml
-        };
-    
-        if ($rec_id) {
-            # Let other backends grab our data now that we're done.
-            my $key = 'TPAC_unapi_cache_'.$rec_id.'_'.$unapi_cache_key_suffix;
-            my $cache_data = $unapi_cache->get_cache($key);
-            if ($$cache_data{running}) {
-                $unapi_cache->put_cache($key, {
-                    bre_id => $bre_id,
-                    mmr_id => $mmr_id,
-                    id => $rec_id, 
-                    marc_xml => $xml->toString
-                }, 10);
-            }
-        }
-    }
 
     return ($facets, map { $tmp_data{$_} } @$rec_ids);
 }