From 8f51c52e41bf3dcf7141d8328b522e2a215ccdb8 Mon Sep 17 00:00:00 2001
From: gmc <gmc@dcc99617-32d9-48b4-a31d-7c20da2025e4>
Date: Mon, 22 Nov 2010 13:17:57 +0000
Subject: [PATCH] parallel fine generator
The fine generator cronjob can now use multiple
parallel processes by setting fine_generator/parallel
in opensrf.xml to a value greater than 1. This
can speed up periodic fine generation in a database
containing a large number of overdue loans.
Also added a service to return just the list of
IDs of overdue loans and reservations - fleshing
the entire set of overdue loans when generating fines
has been observed to cause significant swap-thrashing in
at least one large database.
Signed-off-by: Galen Charlton <gmc@esilibrary.com>
git-svn-id: svn://svn.open-ils.org/ILS/trunk@18817 dcc99617-32d9-48b4-a31d-7c20da2025e4
---
Open-ILS/examples/opensrf.xml.example | 6 ++++
.../Application/Storage/Publisher/action.pm | 15 ++++++++--
Open-ILS/src/support-scripts/fine_generator.pl | 34 +++++++++++++++++++---
3 files changed, 48 insertions(+), 7 deletions(-)
diff --git a/Open-ILS/examples/opensrf.xml.example b/Open-ILS/examples/opensrf.xml.example
index 464592993d..ac959c3ca7 100644
--- a/Open-ILS/examples/opensrf.xml.example
+++ b/Open-ILS/examples/opensrf.xml.example
@@ -139,6 +139,12 @@ vim:et:ts=4:sw=4:
-->
<parallel>1</parallel>
</hold_targeter>
+
+ <!-- Settings for the fine generator cron job -->
+ <fine_generator>
+ <!-- number of parallel processes to use during fine generation -->
+ <parallel>1</parallel>
+ </fine_generator>
<reporter>
<!--
diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/action.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/action.pm
index 25f9cd024e..7697937f2a 100644
--- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/action.pm
+++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/action.pm
@@ -97,6 +97,7 @@ __PACKAGE__->register_method(
sub overdue_circs {
my $grace = shift;
my $upper_interval = shift || '1 millennium';
+ my $idlist = shift;
my $c_t = action::circulation->table;
@@ -117,7 +118,7 @@ sub overdue_circs {
my $sth = action::circulation->db_Main->prepare_cached($sql);
$sth->execute($upper_interval);
- my @circs = map { action::circulation->construct($_) } $sth->fetchall_hash;
+ my @circs = map { $idlist ? $_->{id} : action::circulation->construct($_) } $sth->fetchall_hash;
$c_t = booking::reservation->table;
$sql = <<" SQL";
@@ -132,7 +133,7 @@ sub overdue_circs {
$sth = action::circulation->db_Main->prepare_cached($sql);
$sth->execute();
- push @circs, map { booking::reservation->construct($_) } $sth->fetchall_hash;
+ push @circs, map { $idlist ? $_->{id} : booking::reservation->construct($_) } $sth->fetchall_hash;
return @circs;
}
@@ -267,7 +268,9 @@ sub grab_overdue {
my $client = shift;
my $grace = shift || '';
- $client->respond( $_->to_fieldmapper ) for ( overdue_circs($grace) );
+ my $idlist = $self->api_name =~/id_list/o ? 1 : 0;
+
+ $client->respond( $idlist ? $_ : $_->to_fieldmapper ) for ( overdue_circs($grace, '', $idlist) );
return undef;
@@ -278,6 +281,12 @@ __PACKAGE__->register_method(
stream => 1,
method => 'grab_overdue',
);
+__PACKAGE__->register_method(
+ api_name => 'open-ils.storage.action.circulation.overdue.id_list',
+ api_level => 1,
+ stream => 1,
+ method => 'grab_overdue',
+);
sub nearest_hold {
my $self = shift;
diff --git a/Open-ILS/src/support-scripts/fine_generator.pl b/Open-ILS/src/support-scripts/fine_generator.pl
index eb62dbdea6..8859b13a70 100755
--- a/Open-ILS/src/support-scripts/fine_generator.pl
+++ b/Open-ILS/src/support-scripts/fine_generator.pl
@@ -8,6 +8,8 @@ use strict;
use warnings;
use OpenSRF::Utils::JSON;
use OpenSRF::System;
+use OpenSRF::Utils::SettingsClient;
+use OpenSRF::MultiSession;
my $config = shift || die "bootstrap config required\n";
my $lockfile = shift || "/tmp/generate_fines-LOCK";
@@ -35,11 +37,35 @@ print F $$;
close F;
OpenSRF::System->bootstrap_client( config_file => $config );
+my $settings = OpenSRF::Utils::SettingsClient->new;
+my $parallel = $settings->config_value( fine_generator => 'parallel' ) || 1;
-my $r = OpenSRF::AppSession
- ->create( 'open-ils.storage' )
- ->request( 'open-ils.storage.action.circulation.overdue.generate_fines' => $grace );
+if ($parallel == 1) {
-while (!$r->complete) { $r->recv };
+ my $r = OpenSRF::AppSession
+ ->create( 'open-ils.storage' )
+ ->request( 'open-ils.storage.action.circulation.overdue.generate_fines' => $grace );
+
+ while (!$r->complete) { $r->recv };
+
+} else {
+
+ my $multi_generator = OpenSRF::MultiSession->new(
+ app => 'open-ils.storage',
+ cap => $parallel,
+ api_level => 1,
+ );
+
+ my $storage = OpenSRF::AppSession->create("open-ils.storage");
+ my $r = $storage->request('open-ils.storage.action.circulation.overdue.id_list', $grace);
+ while (my $resp = $r->recv) {
+ my $circ_id = $resp->content;
+ $multi_generator->request( 'open-ils.storage.action.circulation.overdue.generate_fines', $grace, $circ_id );
+ }
+ $storage->disconnect();
+ $multi_generator->session_wait(1);
+ $multi_generator->disconnect;
+
+}
unlink $lockfile;
--
2.11.0