From d3c2eb86aaf38fe896953ab416e9e9e69aab5b78 Mon Sep 17 00:00:00 2001
From: miker <miker@dcc99617-32d9-48b4-a31d-7c20da2025e4>
Date: Thu, 4 Dec 2008 05:30:32 +0000
Subject: [PATCH] queue up ingest requests in a secondary stream to avoid
 piling them up and exhausting cstore sessions

git-svn-id: svn://svn.open-ils.org/ILS/trunk@11401 dcc99617-32d9-48b4-a31d-7c20da2025e4
---
 .../perlmods/OpenILS/Application/Cat/BibCommon.pm  | 20 +++++++++-------
 .../src/perlmods/OpenILS/Application/Vandelay.pm   | 27 ++++++++++++++++++----
 2 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Cat/BibCommon.pm b/Open-ILS/src/perlmods/OpenILS/Application/Cat/BibCommon.pm
index 183b946b43..658e73f031 100644
--- a/Open-ILS/src/perlmods/OpenILS/Application/Cat/BibCommon.pm
+++ b/Open-ILS/src/perlmods/OpenILS/Application/Cat/BibCommon.pm
@@ -36,7 +36,7 @@ sub fetch_bib_sources {
 
 
 sub biblio_record_replace_marc  {
-	my($class, $e, $recid, $newxml, $source, $fixtcn, $override) = @_;
+	my($class, $e, $recid, $newxml, $source, $fixtcn, $override, $noingest) = @_;
 
 	my $rec = $e->retrieve_biblio_record_entry($recid)
 		or return $e->die_event;
@@ -70,15 +70,17 @@ sub biblio_record_replace_marc  {
 	$rec->marc( $U->entityize( $marcdoc->documentElement->toString ) );
 	$e->update_biblio_record_entry($rec) or return $e->die_event;
 
-    # we don't care about the result, just fire off the request
-    my $ses = OpenSRF::AppSession->create('open-ils.ingest');
-    $ses->request('open-ils.ingest.full.biblio.record', $recid);
+    unless ($noingest) {
+        # we don't care about the result, just fire off the request
+        my $ses = OpenSRF::AppSession->create('open-ils.ingest');
+        $ses->request('open-ils.ingest.full.biblio.record', $recid);
+    }
 
 	return $rec;
 }
 
 sub biblio_record_xml_import {
-	my($class, $e, $xml, $source, $auto_tcn, $override) = @_;
+	my($class, $e, $xml, $source, $auto_tcn, $override, $noingest) = @_;
 
 	my( $evt, $tcn, $tcn_source, $marcdoc );
 
@@ -107,9 +109,11 @@ sub biblio_record_xml_import {
     $record = $e->create_biblio_record_entry($record) or return $e->die_event;
 	$logger->info("marc create/import created new record ".$record->id);
 
-    # we don't care about the result, just fire off the request
-    my $ses = OpenSRF::AppSession->create('open-ils.ingest');
-    $ses->request('open-ils.ingest.full.biblio.record', $record->id);
+    unless ($noingest) {
+        # we don't care about the result, just fire off the request
+        my $ses = OpenSRF::AppSession->create('open-ils.ingest');
+        $ses->request('open-ils.ingest.full.biblio.record', $record->id);
+    }
 
 	return $record;
 }
diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm b/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm
index b0e34e296d..a557e5f56c 100644
--- a/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm
+++ b/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm
@@ -534,6 +534,9 @@ sub import_record_list_impl {
     my $total = @$rec_ids;
     my $count = 0;
     my %queues;
+    my @ingest_queue;
+
+    my $ingest_ses = OpenSRF::AppSession->connect('open-ils.ingest');
 
     for my $rec_id (@$rec_ids) {
 
@@ -562,11 +565,11 @@ sub import_record_list_impl {
             if(defined $overlay_target) {
                 $logger->info("vl: overlaying record $overlay_target");
                 $record = OpenILS::Application::Cat::BibCommon->biblio_record_replace_marc(
-                    $e, $overlay_target, $rec->marc); #$rec->bib_source
+                    $e, $overlay_target, $rec->marc, undef, undef, undef, 1); #$rec->bib_source
             } else {
                 $logger->info("vl: importing new record");
                 $record = OpenILS::Application::Cat::BibCommon->biblio_record_xml_import(
-                    $e, $rec->marc); #$rec->bib_source
+                    $e, $rec->marc, undef, undef, undef, 1); #$rec->bib_source
             }
 
             if($U->event_code($record)) {
@@ -583,6 +586,10 @@ sub import_record_list_impl {
                 next;
             }
 
+            $e->commit;
+            push @ingest_queue, $ingest_ses->request('open-ils.ingest.full.biblio.record', $rec_id);
+            push @ingest_queue, { req => $ingest_ses->request('open-ils.ingest.full.biblio.record', $rec_id), rec_id => $rec_id };
+
         } else { # authority
 
             my $rec = $e->retrieve_vandelay_queued_authority_record($rec_id);
@@ -623,9 +630,11 @@ sub import_record_list_impl {
                 $e->rollback;
                 next;
             }
+
+            $e->commit;
+            push @ingest_queue, { req => $ingest_ses->request('open-ils.ingest.full.authority.record', $rec_id), rec_id => $rec_id };
         }
 
-        $e->commit;
         $conn->respond({total => $total, progress => $count, imported => $rec_id}) if (++$count % 10) == 0;
     }
 
@@ -641,7 +650,7 @@ sub import_record_list_impl {
                     $queue->complete('t');
                     $e->update_vandelay_bib_queue($queue) or return $e->die_event;
                     $e->commit;
-                    return;
+                    last
                 }
             }
         } else {
@@ -653,12 +662,20 @@ sub import_record_list_impl {
                     $queue->complete('t');
                     $e->update_vandelay_authority_queue($queue) or return $e->die_event;
                     $e->commit;
-                    return;
+                    last
                 }
             }
         }
     }
 
+    $count = 0;
+    for my $ingest (@ingest_queue) {
+        try { $ingest->{req}->gather(1); } otherwise {};
+        $conn->respond({total => $total, progress => $count, imported => $ingest->{rec_id}}) if (++$count % 10) == 0;
+    } 
+
+    $ingest_ses->disconnect();
+
     $e->rollback;
     return undef;
 }
-- 
2.11.0