parallel fine generator

The fine generator cronjob can now use multiple
parallel processes by setting fine_generator/parallel
in opensrf.xml to a value greater than 1.  This
can speed up periodic fine generation in a database
containing a large number of overdue loans.

Also added a service to return just the list of
IDs of overdue loans and reservations - fleshing
the entire set of overdue loans when generating fines
has been observed to cause significant swap-thrashing in
at least one large database.

Signed-off-by: Galen Charlton <gmc@esilibrary.com>
git-svn-id: svn://svn.open-ils.org/ILS/trunk@18817 dcc99617-32d9-48b4-a31d-7c20da2025e4

diff --git a/Open-ILS/examples/opensrf.xml.example b/Open-ILS/examples/opensrf.xml.example
index 4645929..ac959c3 100644 (file)
--- a/Open-ILS/examples/opensrf.xml.example
+++ b/Open-ILS/examples/opensrf.xml.example
@@ -139,6 +139,12 @@ vim:et:ts=4:sw=4:
             -->
             <parallel>1</parallel>
         </hold_targeter>
+
+        <!-- Settings for the fine generator cron job -->
+        <fine_generator>
+            <!-- number of parallel processes to use during fine generation -->
+            <parallel>1</parallel>
+        </fine_generator>
         
         <reporter>
             <!--

diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/action.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/action.pm
index 25f9cd0..7697937 100644 (file)
--- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/action.pm
+++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/action.pm
@@ -97,6 +97,7 @@ __PACKAGE__->register_method(
 sub overdue_circs {
        my $grace = shift;
     my $upper_interval = shift || '1 millennium';
+       my $idlist = shift;
 
        my $c_t = action::circulation->table;
 
@@ -117,7 +118,7 @@ sub overdue_circs {
        my $sth = action::circulation->db_Main->prepare_cached($sql);
        $sth->execute($upper_interval);
 
-       my @circs = map { action::circulation->construct($_) } $sth->fetchall_hash;
+       my @circs = map { $idlist ? $_->{id} : action::circulation->construct($_) } $sth->fetchall_hash;
 
        $c_t = booking::reservation->table;
        $sql = <<"      SQL";
@@ -132,7 +133,7 @@ sub overdue_circs {
        $sth = action::circulation->db_Main->prepare_cached($sql);
        $sth->execute();
 
-    push @circs, map { booking::reservation->construct($_) } $sth->fetchall_hash;
+    push @circs, map { $idlist ? $_->{id} : booking::reservation->construct($_) } $sth->fetchall_hash;
 
     return @circs;
 }
@@ -267,7 +268,9 @@ sub grab_overdue {
        my $client = shift;
        my $grace = shift || '';
 
-       $client->respond( $_->to_fieldmapper ) for ( overdue_circs($grace) );
+       my $idlist = $self->api_name =~/id_list/o ? 1 : 0;
+    
+       $client->respond( $idlist ? $_ : $_->to_fieldmapper ) for ( overdue_circs($grace, '', $idlist) );
 
        return undef;
 
@@ -278,6 +281,12 @@ __PACKAGE__->register_method(
        stream          => 1,
        method          => 'grab_overdue',
 );
+__PACKAGE__->register_method(
+       api_name        => 'open-ils.storage.action.circulation.overdue.id_list',
+       api_level       => 1,
+       stream          => 1,
+       method          => 'grab_overdue',
+);
 
 sub nearest_hold {
        my $self = shift;

diff --git a/Open-ILS/src/support-scripts/fine_generator.pl b/Open-ILS/src/support-scripts/fine_generator.pl
index eb62dbd..8859b13 100755 (executable)
--- a/Open-ILS/src/support-scripts/fine_generator.pl
+++ b/Open-ILS/src/support-scripts/fine_generator.pl
@@ -8,6 +8,8 @@ use strict;
 use warnings;
 use OpenSRF::Utils::JSON;
 use OpenSRF::System;
+use OpenSRF::Utils::SettingsClient;
+use OpenSRF::MultiSession;
 
 my $config = shift || die "bootstrap config required\n";
 my $lockfile = shift || "/tmp/generate_fines-LOCK";
@@ -35,11 +37,35 @@ print F $$;
 close F;
 
 OpenSRF::System->bootstrap_client( config_file => $config );
+my $settings = OpenSRF::Utils::SettingsClient->new;
+my $parallel = $settings->config_value( fine_generator => 'parallel' ) || 1; 
 
-my $r = OpenSRF::AppSession
-               ->create( 'open-ils.storage' )
-               ->request( 'open-ils.storage.action.circulation.overdue.generate_fines' => $grace );
+if ($parallel == 1) {
 
-while (!$r->complete) { $r->recv };
+    my $r = OpenSRF::AppSession
+            ->create( 'open-ils.storage' )
+            ->request( 'open-ils.storage.action.circulation.overdue.generate_fines' => $grace );
+
+    while (!$r->complete) { $r->recv };
+
+} else {
+
+    my $multi_generator = OpenSRF::MultiSession->new(
+        app => 'open-ils.storage', 
+        cap => $parallel, 
+        api_level => 1,
+    );
+
+    my $storage = OpenSRF::AppSession->create("open-ils.storage");
+    my $r = $storage->request('open-ils.storage.action.circulation.overdue.id_list', $grace);
+    while (my $resp = $r->recv) {
+        my $circ_id = $resp->content;
+        $multi_generator->request( 'open-ils.storage.action.circulation.overdue.generate_fines', $grace, $circ_id );
+    }
+    $storage->disconnect();
+    $multi_generator->session_wait(1);
+    $multi_generator->disconnect;
+
+}
 
 unlink $lockfile;