From 8cefd1977a5080b6458671872dcda62a38f72aec Mon Sep 17 00:00:00 2001 From: Bill Erickson Date: Thu, 30 Jun 2016 17:40:55 -0400 Subject: [PATCH] LP#1596595 hold_targeter_v2.pl, parallel, new options Move new targeter to Open-ILS/src/support-scripts/hold_targeter_v2.pl and recover existing hold targeter for backwards-compat. Add options to hold targeter v2: --verbose Print process counts --parallel Number of parallel hold processors to run. This overrides any value found in opensrf.xml --target-all Target all active holds, regardless of when they were last targeted. --skip-viable Avoid modifying holds that currently target viable copies. In other words, only (re)target holds in a non-viable state. --retarget-interval Override the 'circ.holds.retarget_interval' global_flag value. --parallel-init-sleep Time to wait between starting each parallel instance. Useful for avoiding dog-piling the DB. Signed-off-by: Bill Erickson --- Open-ILS/src/Makefile.am | 1 + Open-ILS/src/support-scripts/hold_targeter.pl | 110 +++++++++---- Open-ILS/src/support-scripts/hold_targeter_v2.pl | 191 +++++++++++++++++++++++ 3 files changed, 274 insertions(+), 28 deletions(-) mode change 100755 => 100644 Open-ILS/src/support-scripts/hold_targeter.pl create mode 100755 Open-ILS/src/support-scripts/hold_targeter_v2.pl diff --git a/Open-ILS/src/Makefile.am b/Open-ILS/src/Makefile.am index 00740f368b..09a643928c 100644 --- a/Open-ILS/src/Makefile.am +++ b/Open-ILS/src/Makefile.am @@ -60,6 +60,7 @@ core_data = @srcdir@/extras/ils_events.xml \ core_scripts = $(examples)/oils_ctl.sh \ $(supportscr)/fine_generator.pl \ $(supportscr)/hold_targeter.pl \ + $(supportscr)/hold_targeter_v2.pl \ $(supportscr)/reshelving_complete.srfsh \ $(supportscr)/clear_expired_circ_history.srfsh \ $(supportscr)/update_hard_due_dates.srfsh \ diff --git a/Open-ILS/src/support-scripts/hold_targeter.pl b/Open-ILS/src/support-scripts/hold_targeter.pl old mode 100755 new mode 100644 index d6549ea553..2ca196e1ae --- a/Open-ILS/src/support-scripts/hold_targeter.pl +++ b/Open-ILS/src/support-scripts/hold_targeter.pl @@ -1,45 +1,99 @@ #!/usr/bin/perl +# --------------------------------------------------------------------- +# Usage: +# hold_targeter.pl +# --------------------------------------------------------------------- + use strict; use warnings; +use OpenSRF::Utils::JSON; use OpenSRF::System; -use OpenILS::Utils::Fieldmapper; -use OpenILS::Utils::HoldTargeter; -#---------------------------------------------------------------- -# Batch hold (re)targeter -# -# Usage: -# ./hold_targeter.pl /openils/conf/opensrf_core.xml -#---------------------------------------------------------------- +use OpenSRF::Utils::SettingsClient; +use OpenSRF::MultiSession; +use OpenSRF::EX qw(:try); -my $osrf_config = shift || '/openils/conf/opensrf_core.xml'; +my $config = shift || die "bootstrap config required\n"; my $lockfile = shift || "/tmp/hold_targeter-LOCK"; -die "I seem to be running already. If not remove $lockfile, try again\n" - if -e $lockfile; +if (-e $lockfile) { + die "I seem to be running already. If not remove $lockfile, try again\n"; +} -open(LOCK, ">$lockfile") or die "Cannot open lock file: $lockfile : $@\n"; -print LOCK $$ or die "Cannot write to lock file: $lockfile : $@\n"; -close LOCK; +open(F, ">$lockfile"); +print F $$; +close F; -eval { # Make sure we can delete the lock file. +my $settings; +my $parallel; - OpenSRF::System->bootstrap_client(config_file => $osrf_config); - Fieldmapper->import( - IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL")); - OpenILS::Utils::CStoreEditor::init(); +try { + OpenSRF::System->bootstrap_client( config_file => $config ); + $settings = OpenSRF::Utils::SettingsClient->new; + $parallel = $settings->config_value( hold_targeter => 'parallel' ) || 1; +} otherwise { + my $e = shift; + warn "$e\n"; + unlink $lockfile; + exit 1; +}; - my $targeter = OpenILS::Utils::HoldTargeter->new; +if ($parallel == 1) { - my $start = time; - my $count = $targeter->target( - #skip_viable => 1, # Testing: only re-target non-viable holds. - return_count => 1 # Return count, not per-hold results. - ); + try { + my $r = OpenSRF::AppSession + ->create( 'open-ils.storage' ) + ->request( 'open-ils.storage.action.hold_request.copy_targeter' => '24h' ); - my $minutes = sprintf('%0.2f', (time - $start) / 60.0); + while (!$r->complete) { + my $start = time; + $r->recv(timeout => 3600); + last if (time() - $start) >= 3600; + }; + } otherwise { + my $e = shift; + warn "Failure in single-session targeter:\n$e\n"; + }; - print "Processed $count holds in $minutes minutes.\n"; -}; +} else { + + try { + my $multi_targeter = OpenSRF::MultiSession->new( + app => 'open-ils.storage', + cap => $parallel, + api_level => 1, + session_hash_function => sub { + my $ses = shift; + my $req = shift; + return $_[-1]; # last parameter is the ID of the metarecord associated with the + # request's target; using this as the hash function value ensures + # that parallel targeters won't try to simultaneously handle two + # hold requests that have overlapping pools of copies that could + # fill those requests + } + ); + + my $storage = OpenSRF::AppSession->create("open-ils.storage"); + + my $r = $storage->request('open-ils.storage.action.hold_request.targetable_holds.id_list', '24h'); + while ( my $h = $r->recv ) { + if ($r->failed) { + print $r->failed->stringify . "\n"; + last; + } + if (my $hold = $h->content) { + $multi_targeter->request( 'open-ils.storage.action.hold_request.copy_targeter', '', $hold->[0], $hold->[1]); + } + } + + $storage->disconnect(); + + $multi_targeter->session_wait(1); + $multi_targeter->disconnect; + } otherwise { + my $e = shift; + warn "Failure in multi-session targeter:\n$e\n"; + } +} unlink $lockfile; diff --git a/Open-ILS/src/support-scripts/hold_targeter_v2.pl b/Open-ILS/src/support-scripts/hold_targeter_v2.pl new file mode 100755 index 0000000000..7f342ef6cf --- /dev/null +++ b/Open-ILS/src/support-scripts/hold_targeter_v2.pl @@ -0,0 +1,191 @@ +#!/usr/bin/perl +use strict; +use warnings; +use Getopt::Long; +use OpenSRF::System; +use OpenSRF::AppSession; +use OpenSRF::Utils::SettingsClient; +use OpenILS::Utils::Fieldmapper; +#---------------------------------------------------------------- +# Batch hold (re)targeter +# +# Usage: +# ./hold_targeter.pl /openils/conf/opensrf_core.xml +#---------------------------------------------------------------- + +my $help; +my $osrf_config = '/openils/conf/opensrf_core.xml'; +my $lockfile = '/tmp/hold_targeter-LOCK'; +my $parallel = 0; +my $verbose = 0; +my $target_all; +my $skip_viable; +my $retarget_interval; +my $recv_timeout = 3600; +my $parallel_init_sleep = 0; + +# how often the server sends a summary reply per backend. +my $return_throttle = 50; + +GetOptions( + 'osrf-config=s' => \$osrf_config, + 'lockfile=s' => \$lockfile, + 'parallel=i' => \$parallel, + 'verbose' => \$verbose, + 'target-all' => \$target_all, + 'skip-viable' => \$skip_viable, + 'retarget-interval' => \$retarget_interval, + 'parallel-init-sleep=i' => \$parallel_init_sleep, + 'help' => \$help +) || die "\nSee --help for more\n"; + +sub help { + print < + Number of parallel hold processors to run. This overrides any + value found in opensrf.xml + + --parallel-init-sleep + Number of seconds to wait before starting each subsequent + parallel targeter instance. This gives each targeter backend + time to run the large targetable holds query before the next + kicks off, so they don't all hit the database at once. + + Defaults to no sleep. + + --target-all + Target all active holds, regardless of when they were last targeted. + + --skip-viable + Avoid modifying holds that currently target viable copies. In + other words, only (re)target holds in a non-viable state. + + --retarget-interval + Override the 'circ.holds.retarget_interval' global_flag value. + +HELP + + exit(0); +} + +help() if $help; + +sub init { + + OpenSRF::System->bootstrap_client(config_file => $osrf_config); + Fieldmapper->import( + IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL")); + + if (!$parallel) { + my $settings = OpenSRF::Utils::SettingsClient->new; + $parallel = $settings->config_value(hold_targeter => 'parallel') || 1; + } +} + +sub run_batches { + + # Hanging all of the parallel requests off the same app session + # lets us operate the same as a MultiSession batch with additional + # fine-grained controls over the receive timeout and real-time + # response handling. + my $ses = OpenSRF::AppSession->create('open-ils.hold-targeter'); + + my @reqs; + for my $slot (1..$parallel) { + + if ($slot > 1 && $parallel_init_sleep) { + $verbose && print "Sleeping $parallel_init_sleep ". + "seconds before targeter slot=$slot launch\n"; + sleep $parallel_init_sleep; + } + + $verbose && print "Starting targeter slot=$slot\n"; + + my $req = $ses->request( + 'open-ils.hold-targeter.target', { + return_count => 1, + return_throttle => $return_throttle, + parallel_count => $parallel, + parallel_slot => $slot, + skip_viable => $skip_viable, + target_all => $target_all, + retarget_interval => $retarget_interval + } + ); + + $req->{_parallel_slot} = $slot; # for grouping/logging below + push(@reqs, $req); + } + + while (@reqs) { + my $start = time; + $ses->queue_wait($recv_timeout); # wait for a response + + # As a fail-safe, exit if no responses have arrived + # within the timeout interval. + last if (time - $start) >= $recv_timeout; + + for my $req (@reqs) { + # Pull all responses off the receive queues. + while (my $resp = $req->recv(0)) { + $verbose && print sprintf( + "Targeter [%d] processed %d holds\n", + $req->{_parallel_slot}, + $resp->content + ); + } + } + + @reqs = grep {!$_->complete} @reqs; + } +} + +# ---- + +die "I seem to be running already. If not remove $lockfile, try again\n" + if -e $lockfile; + +open(LOCK, ">$lockfile") or die "Cannot open lock file: $lockfile : $@\n"; +print LOCK $$ or die "Cannot write to lock file: $lockfile : $@\n"; +close LOCK; + +eval { # Make sure we can delete the lock file. + + init(); + + my $start = time; + + run_batches(); + + my $minutes = sprintf('%0.2f', (time - $start) / 60.0); + + $verbose && print "Processing took $minutes minutes.\n"; +}; + +warn "Hold processing exited with error: $@\n" if $@; + +unlink $lockfile; + -- 2.11.0